{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyalink.alink import *\n",
    "useLocalEnv(1)\n",
    "\n",
    "from utils import *\n",
    "import os\n",
    "import pandas as pd\n",
    "\n",
    "DATA_DIR = ROOT_DIR + \"german_credit\" + os.sep\n",
    "\n",
    "ORIGIN_FILE = \"german.data\";\n",
    "\n",
    "TRAIN_FILE = \"train.ak\";\n",
    "TEST_FILE = \"test.ak\";\n",
    "\n",
    "COL_NAMES = [\n",
    "    \"status\", \"duration\", \"credit_history\", \"purpose\", \"credit_amount\",\n",
    "    \"savings\", \"employment\", \"installment_rate\", \"marriage_sex\", \"debtors\",\n",
    "    \"residence\", \"property\", \"age\", \"other_plan\", \"housing\",\n",
    "    \"number_credits\", \"job\", \"maintenance_num\", \"telephone\", \"foreign_worker\",\n",
    "    \"class\"\n",
    "]\n",
    "\n",
    "COL_TYPES = [\n",
    "    \"string\", \"int\", \"string\", \"string\", \"int\",\n",
    "    \"string\", \"string\", \"int\", \"string\", \"string\",\n",
    "    \"int\", \"string\", \"int\", \"string\", \"string\",\n",
    "    \"int\", \"string\", \"int\", \"string\", \"string\",\n",
    "    \"int\"\n",
    "]\n",
    "\n",
    "CLAUSE_CREATE_FEATURES = \"(case status when 'A11' then 1 else 0 end) as status_A11,\"\\\n",
    "+ \"(case status when 'A12' then 1 else 0 end) as status_A12,\"\\\n",
    "+ \"(case status when 'A13' then 1 else 0 end) as status_A13,\"\\\n",
    "+ \"(case status when 'A14' then 1 else 0 end) as status_A14,\"\\\n",
    "+ \"duration,\"\\\n",
    "+ \"(case credit_history when 'A30' then 1 else 0 end) as credit_history_A30,\"\\\n",
    "+ \"(case credit_history when 'A31' then 1 else 0 end) as credit_history_A31,\"\\\n",
    "+ \"(case credit_history when 'A32' then 1 else 0 end) as credit_history_A32,\"\\\n",
    "+ \"(case credit_history when 'A33' then 1 else 0 end) as credit_history_A33,\"\\\n",
    "+ \"(case credit_history when 'A34' then 1 else 0 end) as credit_history_A34,\"\\\n",
    "+ \"(case purpose when 'A40' then 1 else 0 end) as purpose_A40,\"\\\n",
    "+ \"(case purpose when 'A41' then 1 else 0 end) as purpose_A41,\"\\\n",
    "+ \"(case purpose when 'A42' then 1 else 0 end) as purpose_A42,\"\\\n",
    "+ \"(case purpose when 'A43' then 1 else 0 end) as purpose_A43,\"\\\n",
    "+ \"(case purpose when 'A44' then 1 else 0 end) as purpose_A44,\"\\\n",
    "+ \"(case purpose when 'A45' then 1 else 0 end) as purpose_A45,\"\\\n",
    "+ \"(case purpose when 'A46' then 1 else 0 end) as purpose_A46,\"\\\n",
    "+ \"(case purpose when 'A47' then 1 else 0 end) as purpose_A47,\"\\\n",
    "+ \"(case purpose when 'A48' then 1 else 0 end) as purpose_A48,\"\\\n",
    "+ \"(case purpose when 'A49' then 1 else 0 end) as purpose_A49,\"\\\n",
    "+ \"(case purpose when 'A410' then 1 else 0 end) as purpose_A410,\"\\\n",
    "+ \"credit_amount,\"\\\n",
    "+ \"(case savings when 'A61' then 1 else 0 end) as savings_A61,\"\\\n",
    "+ \"(case savings when 'A62' then 1 else 0 end) as savings_A62,\"\\\n",
    "+ \"(case savings when 'A63' then 1 else 0 end) as savings_A63,\"\\\n",
    "+ \"(case savings when 'A64' then 1 else 0 end) as savings_A64,\"\\\n",
    "+ \"(case savings when 'A65' then 1 else 0 end) as savings_A65,\"\\\n",
    "+ \"(case employment when 'A71' then 1 else 0 end) as employment_A71,\"\\\n",
    "+ \"(case employment when 'A72' then 1 else 0 end) as employment_A72,\"\\\n",
    "+ \"(case employment when 'A73' then 1 else 0 end) as employment_A73,\"\\\n",
    "+ \"(case employment when 'A74' then 1 else 0 end) as employment_A74,\"\\\n",
    "+ \"(case employment when 'A75' then 1 else 0 end) as employment_A75,\"\\\n",
    "+ \"installment_rate,\"\\\n",
    "+ \"(case marriage_sex when 'A91' then 1 else 0 end) as marriage_sex_A91,\"\\\n",
    "+ \"(case marriage_sex when 'A92' then 1 else 0 end) as marriage_sex_A92,\"\\\n",
    "+ \"(case marriage_sex when 'A93' then 1 else 0 end) as marriage_sex_A93,\"\\\n",
    "+ \"(case marriage_sex when 'A94' then 1 else 0 end) as marriage_sex_A94,\"\\\n",
    "+ \"(case marriage_sex when 'A95' then 1 else 0 end) as marriage_sex_A95,\"\\\n",
    "+ \"(case debtors when 'A101' then 1 else 0 end) as debtors_A101,\"\\\n",
    "+ \"(case debtors when 'A102' then 1 else 0 end) as debtors_A102,\"\\\n",
    "+ \"(case debtors when 'A103' then 1 else 0 end) as debtors_A103,\"\\\n",
    "+ \"residence,\"\\\n",
    "+ \"(case property when 'A121' then 1 else 0 end) as property_A121,\"\\\n",
    "+ \"(case property when 'A122' then 1 else 0 end) as property_A122,\"\\\n",
    "+ \"(case property when 'A123' then 1 else 0 end) as property_A123,\"\\\n",
    "+ \"(case property when 'A124' then 1 else 0 end) as property_A124,\"\\\n",
    "+ \"age,\"\\\n",
    "+ \"(case other_plan when 'A141' then 1 else 0 end) as other_plan_A141,\"\\\n",
    "+ \"(case other_plan when 'A142' then 1 else 0 end) as other_plan_A142,\"\\\n",
    "+ \"(case other_plan when 'A143' then 1 else 0 end) as other_plan_A143,\"\\\n",
    "+ \"(case housing when 'A151' then 1 else 0 end) as housing_A151,\"\\\n",
    "+ \"(case housing when 'A152' then 1 else 0 end) as housing_A152,\"\\\n",
    "+ \"(case housing when 'A153' then 1 else 0 end) as housing_A153,\"\\\n",
    "+ \"number_credits,\"\\\n",
    "+ \"(case job when 'A171' then 1 else 0 end) as job_A171,\"\\\n",
    "+ \"(case job when 'A172' then 1 else 0 end) as job_A172,\"\\\n",
    "+ \"(case job when 'A173' then 1 else 0 end) as job_A173,\"\\\n",
    "+ \"(case job when 'A174' then 1 else 0 end) as job_A174,\"\\\n",
    "+ \"maintenance_num,\"\\\n",
    "+ \"(case telephone when 'A192' then 1 else 0 end) as telephone,\"\\\n",
    "+ \"(case foreign_worker when 'A201' then 1 else 0 end) as foreign_worker,\"\\\n",
    "+ \"class \"\n",
    "\n",
    "LABEL_COL_NAME = \"class\";\n",
    "\n",
    "FEATURE_COL_NAMES = COL_NAMES.copy()\n",
    "FEATURE_COL_NAMES.remove(LABEL_COL_NAME)\n",
    "\n",
    "NUMERIC_FEATURE_COL_NAMES = [\n",
    "    \"duration\", \"credit_amount\", \"installment_rate\", \"residence\", \"age\", \"number_credits\", \"maintenance_num\"\n",
    "]\n",
    "\n",
    "CATEGORY_FEATURE_COL_NAMES = FEATURE_COL_NAMES.copy()\n",
    "for numeric_col in NUMERIC_FEATURE_COL_NAMES :\n",
    "    CATEGORY_FEATURE_COL_NAMES.remove(numeric_col)\n",
    "\n",
    "VEC_COL_NAME = \"vec\";\n",
    "\n",
    "PREDICTION_COL_NAME = \"pred\";\n",
    "\n",
    "PRED_DETAIL_COL_NAME = \"predinfo\";"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#c_0\n",
    "source = CsvSourceBatchOp()\\\n",
    "    .setFilePath(DATA_DIR + ORIGIN_FILE)\\\n",
    "    .setSchemaStr(generateSchemaString(COL_NAMES, COL_TYPES))\\\n",
    "    .setFieldDelimiter(\" \");\n",
    "\n",
    "source\\\n",
    "    .lazyPrint(5, \"< origin data >\")\\\n",
    "    .lazyPrintStatistics();\n",
    "\n",
    "BatchOperator.execute();\n",
    "\n",
    "splitTrainTestIfNotExist(source, DATA_DIR + TRAIN_FILE, DATA_DIR + TEST_FILE, 0.8);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#c_1\n",
    "train_data = AkSourceBatchOp()\\\n",
    "    .setFilePath(DATA_DIR + TRAIN_FILE)\\\n",
    "    .select(CLAUSE_CREATE_FEATURES);\n",
    "\n",
    "test_data = AkSourceBatchOp()\\\n",
    "    .setFilePath(DATA_DIR + TEST_FILE)\\\n",
    "    .select(CLAUSE_CREATE_FEATURES);\n",
    "\n",
    "new_features = train_data.getColNames()\n",
    "new_features.remove(LABEL_COL_NAME)\n",
    "\n",
    "train_data.lazyPrint(5, \"< new features >\");\n",
    "\n",
    "trainer = LogisticRegressionTrainBatchOp()\\\n",
    "    .setFeatureCols(new_features)\\\n",
    "    .setLabelCol(LABEL_COL_NAME);\n",
    "\n",
    "predictor = LogisticRegressionPredictBatchOp()\\\n",
    "    .setPredictionCol(PREDICTION_COL_NAME)\\\n",
    "    .setPredictionDetailCol(PRED_DETAIL_COL_NAME);\n",
    "\n",
    "train_data.link(trainer);\n",
    "\n",
    "predictor.linkFrom(trainer, test_data);\n",
    "\n",
    "\n",
    "def print_feature_importance(linearModelTrainInfo: LinearModelTrainInfo):\n",
    "    df = pd.DataFrame({'name':linearModelTrainInfo.getColNames(), \n",
    "                       'value':linearModelTrainInfo.getImportance()})\n",
    "    pd.set_option('display.max_rows', 100)\n",
    "    print(df.sort_values(by = ['value'],axis = 0,ascending = False))\n",
    "    pd.set_option('display.max_rows', 10)\n",
    "    \n",
    "    \n",
    "trainer\\\n",
    "    .lazyPrintTrainInfo()\\\n",
    "    .lazyCollectTrainInfo(print_feature_importance)\n",
    "\n",
    "predictor.link(\n",
    "    EvalBinaryClassBatchOp()\\\n",
    "        .setPositiveLabelValueString(\"2\")\\\n",
    "        .setLabelCol(LABEL_COL_NAME)\\\n",
    "        .setPredictionDetailCol(PRED_DETAIL_COL_NAME)\\\n",
    "        .lazyPrintMetrics()\n",
    ");\n",
    "\n",
    "BatchOperator.execute();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#c_2\n",
    "train_data = AkSourceBatchOp().setFilePath(DATA_DIR + TRAIN_FILE).select(CLAUSE_CREATE_FEATURES);\n",
    "test_data = AkSourceBatchOp().setFilePath(DATA_DIR + TEST_FILE).select(CLAUSE_CREATE_FEATURES);\n",
    "\n",
    "new_features = train_data.getColNames()\n",
    "new_features.remove(LABEL_COL_NAME)\n",
    "\n",
    "train_data.lazyPrint(5, \"< new features >\")\n",
    "\n",
    "trainer = LogisticRegressionTrainBatchOp()\\\n",
    "    .setFeatureCols(new_features)\\\n",
    "    .setLabelCol(LABEL_COL_NAME)\\\n",
    "    .setL1(0.01)\n",
    "\n",
    "predictor = LogisticRegressionPredictBatchOp()\\\n",
    "    .setPredictionCol(PREDICTION_COL_NAME)\\\n",
    "    .setPredictionDetailCol(PRED_DETAIL_COL_NAME)\n",
    "\n",
    "train_data.link(trainer);\n",
    "\n",
    "predictor.linkFrom(trainer, test_data);\n",
    "\n",
    "trainer\\\n",
    "    .lazyPrintTrainInfo()\\\n",
    "    .lazyCollectTrainInfo(print_feature_importance)\n",
    "\n",
    "predictor.link(\n",
    "    EvalBinaryClassBatchOp()\\\n",
    "        .setPositiveLabelValueString(\"2\")\\\n",
    "        .setLabelCol(LABEL_COL_NAME)\\\n",
    "        .setPredictionDetailCol(PRED_DETAIL_COL_NAME)\\\n",
    "        .lazyPrintMetrics()\n",
    ");\n",
    "\n",
    "BatchOperator.execute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#c_3_1\n",
    "train_data = AkSourceBatchOp().setFilePath(DATA_DIR + TRAIN_FILE);\n",
    "test_data = AkSourceBatchOp().setFilePath(DATA_DIR + TEST_FILE);\n",
    "\n",
    "pipeline = Pipeline()\\\n",
    "    .add(\n",
    "        OneHotEncoder()\\\n",
    "            .setSelectedCols(CATEGORY_FEATURE_COL_NAMES)\\\n",
    "            .setEncode('VECTOR')\n",
    "    )\\\n",
    "    .add(\n",
    "        VectorAssembler()\\\n",
    "            .setSelectedCols(FEATURE_COL_NAMES)\\\n",
    "            .setOutputCol(VEC_COL_NAME)\n",
    "    )\\\n",
    "    .add(\n",
    "        LogisticRegression()\\\n",
    "            .setVectorCol(VEC_COL_NAME)\\\n",
    "            .setLabelCol(LABEL_COL_NAME)\\\n",
    "            .setPredictionCol(PREDICTION_COL_NAME)\\\n",
    "            .setPredictionDetailCol(PRED_DETAIL_COL_NAME)\n",
    "    );\n",
    "\n",
    "pipeline\\\n",
    "    .fit(train_data)\\\n",
    "    .transform(test_data)\\\n",
    "    .link(\n",
    "        EvalBinaryClassBatchOp()\\\n",
    "            .setPositiveLabelValueString(\"2\")\\\n",
    "            .setLabelCol(LABEL_COL_NAME)\\\n",
    "            .setPredictionDetailCol(PRED_DETAIL_COL_NAME)\\\n",
    "            .lazyPrintMetrics()\n",
    "    );\n",
    "\n",
    "BatchOperator.execute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#c_3_2\n",
    "train_data = AkSourceBatchOp().setFilePath(DATA_DIR + TRAIN_FILE);\n",
    "test_data = AkSourceBatchOp().setFilePath(DATA_DIR + TEST_FILE);\n",
    "\n",
    "pipeline = Pipeline()\\\n",
    "    .add(\n",
    "        FeatureHasher()\\\n",
    "            .setSelectedCols(FEATURE_COL_NAMES)\\\n",
    "            .setCategoricalCols(CATEGORY_FEATURE_COL_NAMES)\\\n",
    "            .setOutputCol(VEC_COL_NAME)\n",
    "    )\\\n",
    "    .add(\n",
    "        LogisticRegression()\\\n",
    "            .setVectorCol(VEC_COL_NAME)\\\n",
    "            .setLabelCol(LABEL_COL_NAME)\\\n",
    "            .setPredictionCol(PREDICTION_COL_NAME)\\\n",
    "            .setPredictionDetailCol(PRED_DETAIL_COL_NAME)\n",
    "    );\n",
    "\n",
    "pipeline\\\n",
    "    .fit(train_data)\\\n",
    "    .transform(test_data)\\\n",
    "    .link(\n",
    "        EvalBinaryClassBatchOp()\\\n",
    "            .setPositiveLabelValueString(\"2\")\\\n",
    "            .setLabelCol(LABEL_COL_NAME)\\\n",
    "            .setPredictionDetailCol(PRED_DETAIL_COL_NAME)\\\n",
    "            .lazyPrintMetrics()\n",
    "    );\n",
    "\n",
    "BatchOperator.execute()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
